#!/usr/bin/env python
# coding: utf-8

# In[5]:


import os
import re
import spacy
import enchant
import openpyxl
import pandas as pd
from pandas import DataFrame

checker = enchant.Dict("en_US")


# In[6]:


class EnDataClear:
    """
    一个英文数据清洗的 工具类
    """
    nlp = spacy.load("en_core_web_md")

    def __init__(self, data_frame):
        """
        初始化 工具类
        :param data_frame: 传入Dataframe对象
        """
        # 删除重复行
        data_frame = data_frame.drop_duplicates(keep='last')
        # 去掉缺失值
        data_frame = data_frame.dropna(subset=['职位', '任职要求'])
        self.df = data_frame

    @staticmethod
    def convert_fullwidth_to_halfWidth(text):
        '''
        全角字符统一为半全角字符
        '''
        halfwidth_chars = "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"

        # 对应的全角字符Unicode范围
        fullwidth_chars = "１２３４５６７８９０ａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ！“＃＄％＆'（）＊＋，－．／：；＜＝＞？＠［＼］＾＿｀｛｜｝～"

        char_mapping = dict()
        for i in range(len(halfwidth_chars)):
            char_mapping[ord(fullwidth_chars[i])] = halfwidth_chars[i]
        return text.translate(char_mapping)

    @staticmethod
    def chinese_punctuation_to_english(text):
        '''
        中文标点符号统一为英文
        '''
        punctuation_mapping = {'，': ',', '。': '.', '？': '?', '！': '!', '；': ';', '：': ':', '“': '"', '”': '"', '‘': "'",
                               '’': "'", '&#8203;``oaicite:{"number":1,"invalid_reason":"Malformed citation 【': '[',
                               '】"}``&#8203;': ']', '（': '(', '）': ')', '《': '<', '》': '>', '、': ',', '……': '...',
                               '·': '.',
                               '——': '-'}
        # 使用正则表达式替换中文标点符号
        for ch, en in punctuation_mapping.items():
            text = re.sub(re.escape(ch), en, text)
        return text

    @staticmethod
    def Abbreviation_replacement(text):
        # 缩写和它们的扩展 字典
        abbreviations = {"can't": 'cannot', "it's": 'it is', "I'm": 'I am', 'gonna': 'going to', 'wanna': 'want to',
                         "shouldn't": 'should not', "didn't": 'did not', "couldn't": 'could not', "doesn't": 'does not',
                         "won't": 'will not', "i'll": 'I will', "you'll": 'you will', "he'll": 'he will',
                         "she'll": 'she will', "we'll": 'we will', "they'll": 'they will', "I've": 'I have',
                         "you've": 'you have', "we've": 'we have', "they've": 'they have', "I'd": 'I would',
                         "you'd": 'you would', "he'd": 'he would', "she'd": 'she would', "we'd": 'we would',
                         "they'd": 'they would', "haven't": 'have not', "hasn't": 'has not', "wouldn't": 'would not',
                         "should've": 'should have', "could've": 'could have', "might've": 'might have',
                         "must've": 'must have', "Can't": 'Cannot', "It's": 'It is', 'Gonna': 'Going to',
                         'Wanna': 'Want to', "Shouldn't": 'Should not', "Didn't": 'Did not', "Couldn't": 'Could not',
                         "Doesn't": 'Does not', "Won't": 'Will not', "I'll": 'I will', "You'll": 'You will',
                         "He'll": 'He will', "She'll": 'She will', "We'll": 'We will', "They'll": 'They will',
                         "You've": 'You have', "We've": 'We have', "They've": 'They have', "You'd": 'You would',
                         "He'd": 'He would', "She'd": 'She would', "We'd": 'We would', "They'd": 'They would',
                         "Haven't": 'Have not', "Hasn't": 'Has not', "Wouldn't": 'Would not',
                         "Should've": 'Should have', "Could've": 'Could have', "Might've": 'Might have',
                         "Must've": 'Must have'}

        # 替换缩写
        for abbreviation, expansion in abbreviations.items():
            # 使用正则表达式确保只替换完整的单词，而不是部分匹配
            text = re.sub(r'\b' + re.escape(abbreviation) + r'\b', expansion, text)
        text = re.sub("(the[\s]+|The[\s]+)?U\.?S\.?A\.?", " America ", text, flags=re.IGNORECASE)
        text = re.sub("(the[\s]+|The[\s]+)?United State(s)?", " America ", text, flags=re.IGNORECASE)
        return text

    @staticmethod
    def check_spelling(text):
        # 将文本分成单词
        words = text.split()
        # 用于存储纠正后的文本
        corrected_text = []
        for word in words:
            if (word.isalpha() and not word.istitle() and word.islower()) or len(word) > 13:
                if checker.check(word):
                    # 如果拼写正确，保留原词
                    corrected_text.append(word)
                else:
                    # 如果拼写错误，尝试获取建议的正确拼写
                    suggestions = checker.suggest(word)
                    if suggestions:
                        corrected_text.append(suggestions[0])  # 使用第一个建议的拼写
                    else:
                        corrected_text.append(word)  # 如果没有建议，保留原词
            else:
                corrected_text.append(word)
        # 将纠正后的单词组合成文本
        corrected_text = ' '.join(corrected_text)
        return corrected_text

    def replace_special_char(self, cleaned_text):
        """
        去除特殊字符
        :param cleaned_text: 
        :return: 
        """
        # 去掉网址
        patter = re.compile(r'[http|https]*://[a-zA-Z0-9.?/&=:]*|www\.[a-zA-Z0-9.?/&=:]*', re.S)
        cleaned_text = re.sub(patter, '', cleaned_text)
        cleaned_text = re.sub(r"\w+\.(edu|org|com|info)", '', cleaned_text)
        # 去掉@开头的邮箱地址
        cleaned_text = re.sub(r'@\w+\.\w+', ' ', cleaned_text)
        # 去除HTML标记
        pattern = re.compile('(<)?(style=)?.*?>')
        cleaned_text = re.sub(pattern, ' ', cleaned_text)
        # 去掉电话号码
        cleaned_text = re.sub("(-)?\d{3,}-\d{3,}(-\d{3,})?", " ", cleaned_text)
        # 去掉编码不正确的字符
        cleaned_text = ''.join([char for char in cleaned_text if ord(char) < 128])
        # 去掉 `*、(s)、's、 - 、' 、 (1)`
        cleaned_text = re.sub("[*]|\(s\)|\'s|\s+-\s+|(\()?\d+\)", " ", cleaned_text)
        # 去掉 `/> 、146293Job`
        cleaned_text = re.sub("/>|Zone\s+\d+(:)?|phone:|TTY:|tty:|\d+[Jj]ob", " ", cleaned_text)

        # 一些其它替换
        cleaned_text = re.sub("401\(k\)", "401k", cleaned_text)
        cleaned_text = re.sub("\Wtodol", "-dollar ", cleaned_text)
        cleaned_text = re.sub("San Francisco(,)?\s+CA", "San Francisco, California", cleaned_text)
        cleaned_text = re.sub("\s+CA(\s+)?", "California", cleaned_text)
        cleaned_text = re.sub("e\.g\.", " eg ", cleaned_text, flags=re.IGNORECASE)
        cleaned_text = re.sub("b\.g\.", " bg ", cleaned_text, flags=re.IGNORECASE)
        cleaned_text = re.sub("D\.C\.", "DC ", cleaned_text, flags=re.IGNORECASE)
        cleaned_text = re.sub("i\.e\.", " ie ", cleaned_text, flags=re.IGNORECASE)
        cleaned_text = re.sub("[Ii]nc\.", " Inc ", cleaned_text, flags=re.IGNORECASE)
        cleaned_text = re.sub('%', " percent ", cleaned_text)
        cleaned_text = re.sub('<', " less than ", cleaned_text)
        cleaned_text = re.sub('<=', " less than or equal ", cleaned_text)
        cleaned_text = re.sub('>=', " greater than or equal", cleaned_text)
        cleaned_text = re.sub('>', " greater than ", cleaned_text)
        cleaned_text = re.sub('=', " equal ", cleaned_text)
        cleaned_text = re.sub(r" (the[\s]+|The[\s]+)?U\.S\.(A)? ", " America ", cleaned_text)
        cleaned_text = re.sub(r"U(\.)?S(\.)?(A)?", " America ", cleaned_text)

        def rule4(patter):
            matched_string = patter.group(0)
            digit = re.findall("\d+", matched_string)[0]
            if matched_string[0].isdigit():
                return digit + " " + matched_string.split(digit)[-1]
            else:
                return matched_string.split(digit)[-1] + " " + digit

        cleaned_text = re.sub("\d+[a-zA-Z]{3,}", rule4, cleaned_text)
        cleaned_text = re.sub("[a-zA-Z]{3,}\d+", rule4, cleaned_text)

        # /
        def rule1(patter):
            matched_string = patter.group(0)
            return matched_string.replace("/", " per ")

        cleaned_text = re.sub("day(s)?/week|\d+(\s)?/year(s)?", rule1, cleaned_text)
        cleaned_text = re.sub("\s+/\s+|/[Oo][Rr]", "/", cleaned_text)

        def rule2(patter):
            matched_string = patter.group(0)
            return matched_string.replace("/", " or ")

        cleaned_text = re.sub("[A-Za-z]+/[A-Za-z]+", rule2, cleaned_text)
        cleaned_text = re.sub("[A-Za-z]+/\d+", rule2, cleaned_text)
        cleaned_text = re.sub("\d+/[A-Za-z]+", rule2, cleaned_text)

        # &
        cleaned_text = re.sub("\s+&(\s)?", " and ", cleaned_text)
        cleaned_text = re.sub("\Wand/\W|\W&/", " and ", cleaned_text)
        # |
        cleaned_text = re.sub("\|", ", ", cleaned_text)
        # ~
        cleaned_text = re.sub("~", " about ", cleaned_text)

        # 美元
        def rule3(patter):
            matched_string = patter.group(0)
            if "." in matched_string:
                if int(matched_string.split(".")[-1]) <= 0:
                    matched_string = re.sub("\.\d+", " ", matched_string)

            matched_string = matched_string.replace("between", "").replace("and", "-").replace("to", "-")
            return " " + matched_string.replace(",", "").replace(" ", "").replace("$", "dollar") + " "

        cleaned_text = re.sub("\$\d+(,)?\d+(\.)?\d+(\s)?-(\s)?\$\d+(,)?\d+(\.)?\d+", rule3, cleaned_text)
        cleaned_text = re.sub("\$\d+(,)?\d+(\.)?\d+(\s)?to(\s)?\$\d+(,)?\d+(\.)?\d+", rule3, cleaned_text)
        cleaned_text = re.sub("between\s+\$\d+(,)?\d+(\.)?\d+(\s+)?and(\s+)?\$\d+(,)?\d+(\.)?\d+", rule3, cleaned_text)
        cleaned_text = re.sub("\$\d+(,)?\d+(\.)?\d+(\s)?", rule3, cleaned_text)

        # 其它特殊
        cleaned_text = re.sub("#.*?\s|@.*?\s|\[\s+[Ll]ink\s+removed\s+]|\(\d+\)|\[\s+Email\s+address\s+blocked\s+]",
                              " ", cleaned_text)

        # 检查连写
        def rule5(patter):
            matched_string = patter.group(0)
            if matched_string.lower() not in ['powerpoint', 'powershell', 'wordpress', 'macbook', 'youtube', 'tiktok',
                                              'macos', 'matlab', 'stata', "iot", 'javascript', 'javavirtualmachine',
                                              'artificialintelligence', 'machinelearning', 'webdevelopment',
                                              'graphicuserinterface', 'versioncontrolsystem',
                                              'objectorientedprogramming', 'dataencryption', 'virtualreality',
                                              'augmentedreality', 'naturallanguageprocessing',
                                              'databasemanagementsystem', 'operatingsystem', 'mobileappdevelopment',
                                              'responsivewebdesign', 'fullstackdevelopment', 'internetofthings',
                                              'bigdataanalytics', 'cloudcomputing', 'neuralnetwork', 'computervision',
                                              'cybersecurity', 'networkprotocol', 'frontenddevelopment',
                                              'backenddevelopment', 'userexperiencedesign', 'agilemethodology',
                                              'scrummaster', 'devops', 'continuousintegration', 'continuousdelivery',
                                              'machinevision', 'humancomputerinteraction', 'softwareengineering',
                                              'cryptocurrency', 'blockchaintechnology', 'quantumcomputing',
                                              'gamedevelopment', 'compilerdesign', 'distributedsystems',
                                              'informationretrieval', 'digitalsignalprocessing', 'humanoidrobotics',
                                              'bioinformatics', 'geneticalgorithms', 'parallelcomputing',
                                              'swarmintelligence', 'fuzzylogic', 'knowledgerepresentation'] and len(
                    matched_string) >= 5:
                matched_string = re.sub("\w+\s[A-Z]",
                                        lambda x: x.group(0).replace(" ", " . ") if not x.group(0).split(" ")[
                                            0].istitle() else x.group(0),
                                        checker.suggest(matched_string)[0]) if not checker.check(
                    matched_string) and checker.suggest(matched_string) else matched_string
            return matched_string

        cleaned_text = re.sub("[A-Z]+[A-Z]+[a-z]+|([A-Z]+)?[a-z]+[A-Z]+([a-z]+)?", rule5, cleaned_text)
        cleaned_text = re.sub("USD|(\s+)?dol\s+|\$", "dollar", cleaned_text)

        # 数字
        cleaned_text = re.sub('(?<=[0-9])\,(?=[0-9])', "", cleaned_text)

        def solve_float(pattern):
            matched_string = pattern.group(0)
            return str(int(eval(matched_string)))

        cleaned_text = re.sub('[0-9]+\.[0-9]+', solve_float, cleaned_text)

        def add_to(patter):
            matched_string = patter.group(0)
            matched_list = matched_string.split(" ")
            matched_list.insert(1, '-')
            return "".join(matched_list)

        cleaned_text = re.sub("dollar\d+\s+dollar\d+", add_to, cleaned_text)

        def pad_str(s):
            return ' ' + s + ' '

        def pad_pattern(pattern):
            matched_string = pattern.group(0)
            return pad_str(matched_string)

        cleaned_text = re.sub('[\!\?\^\+\,\`\;]', pad_pattern, cleaned_text)
        cleaned_text = re.sub('(?i)([a-z]+:([a-z]+| ))', lambda x: x.group(0).replace(":", " : "),
                              cleaned_text)  # (?i)忽略大小写
        cleaned_text = re.sub("[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]\..", lambda x: x.group(0).replace(".", " . "),
                              cleaned_text)

        def quoted_string_parser(pattern):
            string = pattern.group(0)
            parsed = self.nlp(string[1:-1])
            is_meaningful = False
            for token in parsed:
                if len(token.text) > 2 and not token.text.isdigit() and token.has_vector:
                    is_meaningful = True
            if is_meaningful:
                return string
            else:
                return ''

        cleaned_text = re.sub('\".*?\"', quoted_string_parser, cleaned_text)
        cleaned_text = re.sub("\'.*?\'", quoted_string_parser, cleaned_text)
        cleaned_text = re.sub("\(.*?\)", quoted_string_parser, cleaned_text)
        cleaned_text = re.sub("\[.*?\]", quoted_string_parser, cleaned_text)
        cleaned_text = re.sub("\{.*?\}", quoted_string_parser, cleaned_text)
        cleaned_text = re.sub(' s |"', " ", cleaned_text)

        cleaned_text = re.sub("\'|\"|\{|}|\[|]", " ", cleaned_text)
        # 去掉换行 和多余的空格
        cleaned_text = cleaned_text.replace("\\n", " ")
        cleaned_text = cleaned_text.replace("\\", " ")
        cleaned_text = " ".join([word.strip() for word in cleaned_text.split()])
        return cleaned_text

    def fenJu(self, text):
        cleaned_text = re.sub("[^\W]\\n+\w", lambda x: re.sub("\\n+", " . ", x.group(0)), text)
        cleaned_text = re.sub("[^\W]\n+\w", lambda x: re.sub("\n+", " . ", x.group(0)), cleaned_text)
        cleaned_text = re.sub("\W+\\n+.", lambda x: re.sub("\s+", " . ", x.group(0)) if (x.group(0).strip()[0] if len(
            x.group(0).strip()) > 0 else ":") not in [".", ":", "?", "!", ";"] else re.sub("\s+", " ", x.group(0)),
                              cleaned_text)
        cleaned_text = re.sub("\W+\n+.", lambda x: re.sub("\s+", " . ", x.group(0)) if (x.group(0).strip()[0] if len(
            x.group(0).strip()) > 0 else ":") not in [".", ":", "?", "!", ";"] else re.sub("\s+", " ", x.group(0)),
                              cleaned_text)
        return cleaned_text

    def run(self, text, type_):
        """
        文本清洗
        :param text: 
        """
        if type_ == 1:
            text = self.fenJu(text)
        # 中英文 标点符号统一  中文符号转换为英文
        text = self.chinese_punctuation_to_english(text)
        # 全角字符统一为半全角字符
        text = self.convert_fullwidth_to_halfWidth(text.strip())
        # 替换缩写
        text = self.Abbreviation_replacement(text)
        # 去除特殊字符
        text = self.replace_special_char(text)
        # 拼写检查
        text = self.check_spelling(text)
        return text

    def data_clear(self, row):
        """
        清洗文本的工具函数
        :param row: self.df 的每一行 
        :return: 返回处理后的 self.df每一行 
        """
        if row.get("Qualifications"):
            row['任职要求'] = self.run(row['任职要求'] + " . " + row["Qualifications"], 1)
        else:
            row['任职要求'] = self.run(row['任职要求'], 1)
        row['职位'] = self.run(row['职位'], 0)
        return row

    def main_(self) -> DataFrame:
        """
        执行函数
        :return: 文本清洗后的 df 
        """
        self.df.apply(func=self.data_clear, axis=1)
        return self.df


class ZhDataClear(EnDataClear):
    """
    一个中文数据清洗的 工具类
    """

    def __init__(self, data_frame):
        """
        初始化 工具类
        :param data_frame: 传入Dataframe对象
        """
        super().__init__(data_frame)

    @staticmethod
    def public_special_char(cleaned_text):
        cleaned_text = re.sub(r'<', "小于", cleaned_text)
        cleaned_text = re.sub(r'<=|≦', "小于等于", cleaned_text)
        cleaned_text = re.sub(r'>=|≥', "大于等于", cleaned_text)
        cleaned_text = re.sub(r'>', "大于", cleaned_text)
        cleaned_text = re.sub(r'=', "等于", cleaned_text)

        # 使用正则表达式删除以 "@" 开头的邮箱地址
        cleaned_text = re.sub(r'@\w+\.\w+', ' ', cleaned_text)
        patter = re.compile(r'[http|https]*://[a-zA-Z0-9.?/&=:]*|www\.[a-zA-Z0-9.?/&=:]*', re.S)
        cleaned_text = re.sub(patter, '', cleaned_text)
        cleaned_text = re.sub(r"\w+\.(edu|org|com|info)", '', cleaned_text)
        # 去除HTML标记
        pattern = re.compile('(<)?(style=)?.*?>')
        cleaned_text = re.sub(pattern, ' ', cleaned_text)
        # 去掉序号
        cleaned_text = re.sub(r"\s?\d(,|\.)\s", ', ', cleaned_text)
        cleaned_text = re.sub(r'（\d+）', ' ', cleaned_text)
        cleaned_text = re.sub(r'\n+\d+(.)?\.|^\d+(.)?\.', ", ", cleaned_text)
        cleaned_text = re.sub(r"\d+-\w", lambda x: x.group(0)[-1] if not x.group(0)[-1].isdigit() else x.group(0),
                              cleaned_text)
        cleaned_text = re.sub(r'\n+\d.{5}',
                              lambda x: x.group(0) if "year" in x.group(0) or "年" in x.group(0) or "万" in x.group(
                                  0) or "元" in x.group(0) or len(re.findall("\d", x.group(0))) > 2 else re.sub(
                                  r"\d+(\.)?", ", ", x.group(0)).strip(), cleaned_text)
        cleaned_text = re.sub(r'[a-z]\.|\\n|\\r|\\t', ' ', cleaned_text)
        cleaned_text = re.sub(r'[一二三四五六七八九十]、|[一二三四五六七八九十],', ' ', cleaned_text)
        cleaned_text = re.sub(r'（[一二三四五六七八九十]）', ' ', cleaned_text)
        cleaned_text = re.sub(r'\s\d+）', ' ', cleaned_text)
        cleaned_text = re.sub(
            r"\\uf09e|\\uf09f|\\uf0b7|\\x9f|\\u200b|\\u2002|\\uf06c|\\xa0|\\u3000|\\uf0fc|\\uf0d8|\\ufeff|\\u2028|●|▪",
            " ", cleaned_text)
        cleaned_text = re.sub(r'①|②|③|④|⑤|⑥|⑦|⑧|⑨|⑩|Ø', ' ', cleaned_text)

        def check_contain(string_, list_):
            for i in list_:
                if i in string_:
                    return True
            else:
                return False

        def rule1(patten):
            matched_string = patten.group(0)
            filter_list = ["早", "晚", "下午", "上午", "天", "个", "岁", "月", "元", "～", "-", "——", "号", "路", "万",
                           "w"]
            if check_contain(matched_string, filter_list):
                # 如果包含
                pass
            else:
                if len(re.findall("\d", matched_string)) < 2 and "年" not in matched_string:
                    matched_string = re.sub(r"\d\s,", ', ', matched_string)
                    matched_string = re.sub(r"\d\s\.", ', ', matched_string)
                    matched_string = re.sub(r"\d,", ', ', matched_string)
                    matched_string = re.sub(r"\d\.", ', ', matched_string)
                else:
                    if "年" in matched_string:
                        matched_string = re.sub(r"\d\.", ", ", matched_string)
                    if "." in matched_string:
                        matched_string = re.sub(r"\d\.(\d)?(\d)?[^元0-9]", lambda x: ", " + x.group(0)[-1],
                                                matched_string)
                    if re.findall(",\s\d\w", matched_string):
                        matched_string = re.sub(r",\s+\d\w",
                                                lambda x: " " + x.group(0)[-1] if x.group(0)[-1] not in ["早", "晚",
                                                                                                         "下午", "上午",
                                                                                                         "天", "个",
                                                                                                         "岁", "月",
                                                                                                         "元", "～", "-",
                                                                                                         "——", "号",
                                                                                                         "：", "路",
                                                                                                         "年"] else x.group(
                                                    0), matched_string)
            return matched_string

        cleaned_text = re.sub(r".{3}\d.{3}", rule1, cleaned_text)
        cleaned_text = re.sub(r".{5}\d.{7}", rule1, cleaned_text)
        cleaned_text = re.sub(r"\d\.\d+[-~]?\d+周?岁", lambda x: x.group(0)[2:], cleaned_text)

        cleaned_text = re.sub(r"\d,[\u4e00-\u9fa5]", lambda x: ", " + x.group(0)[2:], cleaned_text)
        cleaned_text = re.sub(r"\d\.[\u4e00-\u9fa5]", lambda x: ", " + x.group(0)[2:], cleaned_text)
        cleaned_text = re.sub(r"\d:[\u4e00-\u9fa5]", lambda x: ", " + x.group(0)[2:], cleaned_text)
        cleaned_text = re.sub(r"\d\.\d-\d年", lambda x: ", " + x.group(0)[2:], cleaned_text)
        cleaned_text = re.sub(r"\d、|\d\s、", ', ', cleaned_text)
        cleaned_text = re.sub(r"\s\d,\s?", ', ', cleaned_text)
        cleaned_text = re.sub(r"\d\.\d+[-~]?\d+周?岁", lambda x: x.group(0)[2:], cleaned_text)

        return cleaned_text

    @staticmethod
    def zh_replace_special_char(cleaned_text):
        """
        去除特殊字符
        :param cleaned_text: 
        :return: 
        """
        # 去掉一些特殊字符
        cleaned_text = re.sub(
            r"🍁|🏁|💝|😁|#|\🎖|★|😤|►|◆|🎸|💪|🤔|💎|↓|💜|‧|『|🚁|🔸|」|\$|’|「|@|🎁|☆|⭐|】|\\t|❀|·|»|🏠|”|🔹|→|🌻|🧩|■|【⃣▪|😎|«|´|🏅|️\👏|😊|🏀|🐣|』|🌺|❗|✨|👣|🎻|\^|🈶|＂|≦|\*|\\r|✅|◎|§|∣|‘|💡|◼||🎉|�|🌈|🌸|…|♦|`|🥇|》|🌟|\[|]|\)|➕|}|=>|\{|\(|•|【|】|《|\$|\\|👏|⃣|'|_",
            '', cleaned_text)
        cleaned_text = re.sub("\s+", " ", cleaned_text)
        return cleaned_text

    def run(self, text, type_):
        """
        文本清洗
        :param type_: 
        :param text: 
        """
        # 全角字符统一为半全角字符
        text = self.convert_fullwidth_to_halfWidth(text.strip())
        text = self.public_special_char(text)
        if not len(re.findall('[A-Z]|[a-z]', text)) / len(text) > 0.8:  # 中文的
            # 去除特殊字符
            text = self.zh_replace_special_char(text)
        else:  # 英文的
            if type_ == 1:
                text = self.fenJu(text)
            # 替换缩写
            text = self.Abbreviation_replacement(text)
            # 去除特殊字符
            text = self.replace_special_char(text)
            # 拼写检查
            text = self.check_spelling(text)
        return text

    def data_clear(self, row):
        """
        清洗文本的工具函数
        :param row: self.df 的每一行 
        :return: 返回处理后的 self.df每一行 
        """
        row['任职要求'] = self.run(row['任职要求'], 1)
        row['职位'] = self.run(row['职位'], 0)
        return row

    def main_(self) -> DataFrame:
        """
        执行函数
        :return: 文本清洗后的 df 
        """
        self.df.apply(func=self.data_clear, axis=1)
        return self.df


workbook = openpyxl.load_workbook('../Data/数据汇总.xlsx')
path_ = "../Data/数据清洗1/"
if not os.path.exists(path_):
    os.mkdir(path_)
for sheet_name in workbook.sheetnames:
    save_path = path_ + f"{sheet_name}.csv"
    item = dict()
    print(f"当前网站: {sheet_name}")
    df = pd.read_excel('../Data/数据汇总.xlsx', sheet_name=sheet_name, index_col=0)
    if re.search('[\u4e00-\u9fa5]', sheet_name):  # 中文网站
        # df = ZhDataClear(data_frame=df).main_()
        pass
    else:  # 英文网站
        df = EnDataClear(data_frame=df).main_()
    df.to_csv(save_path)
